﻿namespace TokenReader

open FSharp.Data

// Processes emails from given CSV fales into list(s) of tokens and calculates token occurences table
module TokenReader = 
    // Sample data for CSV reader
    [<Literal>]
    let private file = ".\Samples\inbox.csv";

    // Samples the CSV file
    type private mails = CsvProvider<Sample = file, Separator = ";">
    
    // Parses email text to tokens
    let private GetTokens (text: string) = 
        text.Split([|','; ';'; ' '; '<'; '>'; ')'; '('; ':'; '['; ']'; '/'|])
            |> Seq.map(fun(s) -> s.Trim([|'.'; ' '; '*'; '-'; '-'|]))
            |> Seq.filter(fun(s) -> s <> null && s.Length > 2)
            
    // Returns sequence of tokenized e-mails
    // TODO: might be interesting to try checking each mail property separately and giving more weight to results from headers
    let private GetTokensPerMail (path: string) = 
        let inbox = mails.Load(path);
        
        seq { 
            for mail in inbox.Data do
                let text = mail.text.ToLowerInvariant() + " " + 
                           mail.from.ToLowerInvariant() + " " + 
                           mail.headers.ToLowerInvariant()

                yield GetTokens(text) 
            }
    
    // Counts token occurence table for given token sequence
    let private CountTokens(tokens) =
        tokens 
            |> Seq.groupBy id
            |> Seq.map (fun (word, sq) -> word, Seq.length sq)

    // Returns one dictionary containing density map of all emails
    let GetWordDensityMap(path: string) = 
        let tokens = GetTokensPerMail(path) |> Seq.concat 
        let countedItems = CountTokens(tokens)
        dict<string, int>(countedItems)

    // Returns sequence of dictionaries containing density map for each email separatelly
    let GetWordDensityMapPerRow(path: string) =
        let tokens = GetTokensPerMail(path)
        seq {  
            for mail in tokens do     
                let countedItems = CountTokens(mail)
                yield dict<string, int>(countedItems)
        }